from bs4 import BeautifulSoup as Soup
from langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader

class MyDataLoader:
    def __init__(self, max_depth=20, url: str = "https://python.langchain.com/docs/concepts/lcel/"):
        self.url = url
        self.max_depth = max_depth

    def _load(self):
        self.loader = RecursiveUrlLoader(
            url=self.url, max_depth=self.max_depth, extractor=lambda x: Soup(x, "html.parser").text
        )

    def get_content(self):
        self._load()
        docs = self.loader.load()
        # # 开始加载
        d_sorted = sorted(docs, key=lambda x: x.metadata["source"])
        d_reversed = list(reversed(d_sorted))
        concatenated_content = "\n\n\n --- \n\n\n".join(
            [doc.page_content for doc in d_reversed]
        )
        print("加载完成")
        return concatenated_content